package net;

import fetch.Fetcher;
import org.apache.http.Header;
import org.apache.http.message.BasicHeader;
import util.ExcelUtils;
import util.Regex;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.sql.Timestamp;
import java.util.*;

/**
 * @fileName: SiChuanPurchase
 * @author: Han
 * @date: 2018/5/30 13:14
 * @dscription: TODO
 */
public class SiChuanPurchase {

    public static void main(String... args) {
        SiChuanPurchase siChuanPurchase = new SiChuanPurchase();
        siChuanPurchase.grab();
    }

    private void grab() {

        Set<String> urlSet = new HashSet<>();
        //获取url
        BufferedReader reader = null;
        try {
            reader = new BufferedReader(new InputStreamReader(new FileInputStream("F:/result.txt")));
            String line;
            while ((line = reader.readLine()) != null) {
                urlSet.add(line);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                Objects.requireNonNull(reader).close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        Fetcher fetcher = new Fetcher();
        Header[] headers = new Header[]{
                new BasicHeader("Connection", "keep-alive"),
                new BasicHeader("Cache-Control", "max-age=0"),
                new BasicHeader("Upgrade-Insecure-Requests", "1"),
                new BasicHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36"),
                new BasicHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"),
                new BasicHeader("Accept-Encoding", "gzip, deflate"),
                new BasicHeader("Accept-Language", "zh-CN,zh;q=0.9"),
        };
        List<Map<String, String>> grabList = new ArrayList<>();
        int i = 0;
        Random random = new Random();
        for (String url : urlSet) {
            try {
                String pageContent = fetcher.getPageContentByGet(url, headers, null);
                /*if (!pageContent.contains("采购项目名称")) {
                    continue;
                }*/
                pageContent = pageContent.replace(" ", "");
                Map<String, String> contentMap = Regex.siChuanPurchase(pageContent);
                contentMap.put("来源",url);
                grabList.add(contentMap);
                int interval = random.nextInt(5);
                System.out.println(new Timestamp(System.currentTimeMillis()) + ":第" + i++
                        + "条已完成,距离下次检索还有" + interval + "秒");
                Thread.sleep(1000 * interval);
            } catch (Exception e) {
                e.printStackTrace();
                System.out.println(url + "出错" + e.getMessage());
            }
        }
        //key:sheet名,value:sheet里的内容
        Map<String, List<Map<String, String>>> sheetMap = new HashMap<>(16);
        sheetMap.put("sheet1", grabList);
        ExcelUtils.writeToLocal("F:/采购公告.xls", sheetMap);
    }
}
